{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.3832.65 Safari/537.36'}{'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.8329.1 Safari/537.36'}\n",
      "\n",
      "https://order.mandarake.co.jp/order/listPage/list?page=1&soldOut=1&keyword=%Doraemon\n",
      "https://www.suruga-ya.jp/search?category=&search_word=+Doraemon&adult_s=1&is_marketplace=0&page=1\n",
      "suruga当前共爬取1页数据\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-8:\n",
      "Traceback (most recent call last):\n",
      "  File \"<string>\", line 166, in run\n",
      "  File \"<string>\", line 99, in parse_data\n",
      "IndexError: list index out of range\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-2-9609a69e14ec>\", line 142, in surugaTask\n",
      "  File \"<ipython-input-2-9609a69e14ec>\", line 102, in surugaNotice\n",
      "  File \"<ipython-input-2-9609a69e14ec>\", line 24, in spider\n",
      "  File \"<string>\", line 175, in run\n",
      "  File \"<string>\", line 88, in send\n",
      "  File \"C:\\Users\\zhang_haoyu\\anaconda3\\lib\\smtplib.py\", line 723, in login\n",
      "    (code, resp) = self.auth(\n",
      "  File \"C:\\Users\\zhang_haoyu\\anaconda3\\lib\\smtplib.py\", line 635, in auth\n",
      "    (code, resp) = self.docmd(\"AUTH\", mechanism + \" \" + response)\n",
      "TypeError: can only concatenate str (not \"bytes\") to str\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\zhang_haoyu\\anaconda3\\lib\\threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\zhang_haoyu\\anaconda3\\lib\\threading.py\", line 870, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-2-9609a69e14ec>\", line 145, in surugaTask\n",
      "  File \"<ipython-input-2-9609a69e14ec>\", line 136, in errorNotice\n",
      "  File \"<string>\", line 88, in send\n",
      "  File \"C:\\Users\\zhang_haoyu\\anaconda3\\lib\\smtplib.py\", line 723, in login\n",
      "    (code, resp) = self.auth(\n",
      "  File \"C:\\Users\\zhang_haoyu\\anaconda3\\lib\\smtplib.py\", line 635, in auth\n",
      "    (code, resp) = self.docmd(\"AUTH\", mechanism + \" \" + response)\n",
      "TypeError: can only concatenate str (not \"bytes\") to str\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://order.mandarake.co.jp/order/listPage/list?page=2&soldOut=1&keyword=brand%Doraemon\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import traceback\n",
    "from threading import Thread\n",
    "import Ipynb_importer\n",
    "from newNotice import newNotice\n",
    "from mandarakeNotice import mandarakeNotice\n",
    "from myItem import myItem\n",
    "from myPooledDB import myPooledDB\n",
    "from myPooledDBFuns import myPooledDBFuns\n",
    "from compareItems import compareItems\n",
    "from sendMail import sendMail\n",
    "\n",
    "\n",
    "class overallControl():\n",
    "    def __init__(self):\n",
    "        return None\n",
    "        \n",
    "    def spider(self,website,searchWord,pageNum):\n",
    "        #运行相应网站的爬取函数，得到最新商品信息列表(newProductList)\n",
    "        #if website==1: #设website==1代表爬取Suruga\n",
    "        #if website==2: #设website==2代表爬取mandarake\n",
    "        if website==1:\n",
    "            newNoticeInstance=newNotice()\n",
    "            newProductList=newNoticeInstance.run(searchWord,pageNum)\n",
    "            print('suruga text spider done')\n",
    "            return newProductList\n",
    "        \n",
    "        elif website==2:\n",
    "            mandarakeNoticeInstance=mandarakeNotice()\n",
    "            newProductList=mandarakeNoticeInstance.run(searchWord,pageNum)\n",
    "            print('mandarake text spider done')\n",
    "            return newProductList\n",
    "        \n",
    "    def compareData(self,newProductList,website):\n",
    "        oldProductList=myPooledDBFuns.fetch_all(website)\n",
    "        \n",
    "        #实例化比较类，运行比较函数，返回该实例对象，用以获取实例对象内的相关参数\n",
    "        compareItemsInstance=compareItems()\n",
    "        compareItemsInstance.setOldProductList(oldProductList)\n",
    "        compareItemsInstance.setNewProductList(newProductList)\n",
    "        compareItemsInstance.compareIfSoldOrCutPrice()\n",
    "        compareItemsInstance.compareIfNew()\n",
    "        return compareItemsInstance\n",
    "\n",
    "        \n",
    "    def saveToMySQL(self,compareItemsInstance,website):\n",
    "        newNoticeInstance=newNotice()\n",
    "        #为需要插入列表(insertList)中的商品爬取对应图片的数据流\n",
    "        insertList0=compareItemsInstance.getInsertList()\n",
    "        insertList=newNoticeInstance.downloadImages(insertList0)\n",
    "        if website==1:\n",
    "            print('suruga image spider done')\n",
    "        elif website==2:\n",
    "            print('mandarake image spider done')\n",
    "        #将需要插入列表(insertList)中的商品信息写入数据库中\n",
    "        myPooledDBFuns.insert(website,insertList)\n",
    "        '''\n",
    "        print('\\n'+\"添加的商品信息: \"+'\\n')\n",
    "        for i in range(len(insertList)):\n",
    "            print(\"商品名称: \"+insertList[i].getPname()+'\\n')\n",
    "            print(\"价格: \"+str(insertList[i].getPrice())+'\\n')\n",
    "            print(\"商品网址: \"+insertList[i].getProductURL()+'\\n')\n",
    "            print(\"更新时间: \"+str(insertList[i].getUpdateTime())+'\\n')\n",
    "            print('\\n')\n",
    "        '''\n",
    "        #将需要删除列表(deleteList)中的商品信息从数据库中删除\n",
    "        deleteList=compareItemsInstance.getDeleteList()\n",
    "        myPooledDBFuns.delete(website,deleteList)\n",
    "        '''\n",
    "        print('\\n'+\"售出的商品信息: \"+'\\n')\n",
    "        for i in range(len(deleteList)):\n",
    "            print(\"商品名称: \"+deleteList[i].getPname()+'\\n')\n",
    "            print(\"价格: \"+str(deleteList[i].getPrice())+'\\n')\n",
    "            print(\"商品网址: \"+deleteList[i].getProductURL()+'\\n')\n",
    "            print(\"更新时间: \"+str(deleteList[i].getUpdateTime())+'\\n')\n",
    "            print('\\n')\n",
    "        '''\n",
    "        \n",
    "        \n",
    "    def sendNoticeMail(self,compareItemsInstance,website):\n",
    "        sendMailInstance=sendMail()\n",
    "        #得到降价列表，并发送降价邮件\n",
    "        CutPriceList=compareItemsInstance.getCutPriceList()\n",
    "        if len(CutPriceList)>0:\n",
    "            email_body=sendMailInstance.newArrivalText(CutPriceList)\n",
    "            msg=sendMailInstance.setText(email_body,website+'有商品降价')\n",
    "            sendMailInstance.send(msg)\n",
    "            print(\"发送降价邮件成功\")\n",
    "        #得到上新列表，并发送上新邮件\n",
    "        newArrivalList=compareItemsInstance.getNewArrivalList()\n",
    "        if len(newArrivalList)>0:\n",
    "            email_body=sendMailInstance.newArrivalText(newArrivalList)\n",
    "            msg=sendMailInstance.setText(email_body,website+'有商品上新')\n",
    "            sendMailInstance.send(msg)\n",
    "            print(\"发送上新邮件成功\")\n",
    "        \n",
    "    def surugaNotice(self,overallControlInstance):\n",
    "        totalNewProductList=[]\n",
    "        \n",
    "        searchWord=\"Doraemon\"\n",
    "        pageNum=3\n",
    "        newProductList0=overallControlInstance.spider(1,searchWord,pageNum+1)\n",
    "        totalNewProductList.extend(newProductList0)\n",
    "        \n",
    "        searchWord=\"ultraman\"\n",
    "        pageNum=10\n",
    "        newProductList1=overallControlInstance.spider(1,searchWord,pageNum+1)\n",
    "        totalNewProductList.extend(newProductList1)\n",
    "        \n",
    "        compareItemsInstance=overallControlInstance.compareData(totalNewProductList,\"suruga\")\n",
    "        overallControlInstance.saveToMySQL(compareItemsInstance,\"suruga\")\n",
    "        overallControlInstance.sendNoticeMail(compareItemsInstance,\"suruga\")\n",
    "        print(\"suruga all clear\")\n",
    "    \n",
    "    def mandarakeNotice(self,overallControlInstance):\n",
    "        totalNewProductList=[]\n",
    "        \n",
    "        searchWord=\"Doraemon\"\n",
    "        pageNum=2\n",
    "        newProductList0=overallControlInstance.spider(2,searchWord,pageNum+1)\n",
    "        totalNewProductList.extend(newProductList0)\n",
    "        \n",
    "        searchWord=\"ultraman\"\n",
    "        pageNum=3\n",
    "        newProductList1=overallControlInstance.spider(2,searchWord,pageNum+1)\n",
    "        totalNewProductList.extend(newProductList1)\n",
    "        \n",
    "        compareItemsInstance=overallControlInstance.compareData(totalNewProductList,\"mandarake\")\n",
    "        overallControlInstance.saveToMySQL(compareItemsInstance,\"mandarake\")\n",
    "        overallControlInstance.sendNoticeMail(compareItemsInstance,\"mandarake\")\n",
    "        print(\"mandarake all clear\")\n",
    "    \n",
    "    def errorNotice(self,email_body,subject):\n",
    "        sendMailInstance=sendMail()\n",
    "        msg=sendMailInstance.sendError(email_body,subject)\n",
    "        sendMailInstance.send(msg)\n",
    "        \n",
    "    def surugaTask(self):\n",
    "        overallControlInstance=overallControl()\n",
    "        while(1):\n",
    "            try:\n",
    "                overallControlInstance.surugaNotice(overallControlInstance)\n",
    "            except:\n",
    "                error = traceback.format_exc()\n",
    "                overallControlInstance.errorNotice(error,'suruga运行出错')\n",
    "            \n",
    "            time.sleep(600)\n",
    "        \n",
    "    def mandarakeTask(self):\n",
    "        overallControlInstance=overallControl()\n",
    "        while(1):\n",
    "            try:\n",
    "                overallControlInstance.mandarakeNotice(overallControlInstance)\n",
    "            except:\n",
    "                error = traceback.format_exc()\n",
    "                overallControlInstance.errorNotice(error,'mandarake运行出错')\n",
    "        \n",
    "            time.sleep(3601)\n",
    "        \n",
    "myPooledDB.setPOOL()\n",
    "overallControlInstance=overallControl()\n",
    "surugaTaskInstance=Thread(target=overallControlInstance.surugaTask)\n",
    "mandarakeTaskInstance=Thread(target=overallControlInstance.mandarakeTask)\n",
    "surugaTaskInstance.start()\n",
    "mandarakeTaskInstance.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
